{
 "cells": [
  {
   "cell_type": "markdown",
   "id": "de164fb9",
   "metadata": {},
   "source": [
    "# Run Evaluations on our RAG chatbot! \n",
    "\n",
    "<div align=\"center\">\n",
    "    <p style=\"text-align:left\">\n",
    "        <img alt=\"phoenix logo\" src=\"https://repository-images.githubusercontent.com/564072810/f3666cdf-cb3e-4056-8a25-27cb3e6b5848\" width=\"800\"/>\n",
    "        <br>\n",
    "        <a href=\"https://arize.com/docs/phoenix/\">Docs</a>\n",
    "        |\n",
    "        <a href=\"https://github.com/Arize-ai/phoenix\">GitHub</a>\n",
    "        |\n",
    "        <a href=\"https://arize-ai.slack.com/join/shared_invite/zt-2w57bhem8-hq24MB6u7yE_ZF_ilOYSBw#/shared-invite/email\">Community</a>\n",
    "    </p>\n",
    "</div>\n",
    "\n",
    "## Let's get started! "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "5b9943db",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Note: you may need to restart the kernel to use updated packages.\n"
     ]
    }
   ],
   "source": [
    "%pip install -qqq \"arize-phoenix==11.21.0\" \"openai>=1\" nest_asyncio"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "3b0e3371",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/Users/sriichavali/Desktop/llm-zoomcamp/04-monitoring/dtczoomcamp/lib/python3.13/site-packages/tqdm/auto.py:21: TqdmWarning: IProgress not found. Please update jupyter and ipywidgets. See https://ipywidgets.readthedocs.io/en/stable/user_install.html\n",
      "  from .autonotebook import tqdm as notebook_tqdm\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "from getpass import getpass\n",
    "import phoenix as px\n",
    "import nest_asyncio\n",
    "\n",
    "if not (openai_api_key := os.getenv(\"OPENAI_API_KEY\")):\n",
    "    openai_api_key = getpass(\"🔑 Enter your OpenAI API key: \")\n",
    "os.environ[\"OPENAI_API_KEY\"] = openai_api_key\n",
    "\n",
    "nest_asyncio.apply()"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9a03eb13",
   "metadata": {},
   "source": [
    "<img alt=\"Document Retrieval Evaluation Image\" src=\"https://storage.googleapis.com/arize-phoenix-assets/assets/images/phoenix-docs-images/documentRelevanceDiagram.png\" width=\"1000\"/>"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "b471c7cb",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>context.trace_id</th>\n",
       "      <th>input</th>\n",
       "      <th>reference</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>context.span_id</th>\n",
       "      <th>document_position</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">a11dcc2f37759fa2</th>\n",
       "      <th>0</th>\n",
       "      <td>2f4acf21a7d6e2492d809861978bbf71</td>\n",
       "      <td>course title</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>2f4acf21a7d6e2492d809861978bbf71</td>\n",
       "      <td>course title</td>\n",
       "      <td>Yes, we will keep all the materials after the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>2f4acf21a7d6e2492d809861978bbf71</td>\n",
       "      <td>course title</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>2f4acf21a7d6e2492d809861978bbf71</td>\n",
       "      <td>course title</td>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>2f4acf21a7d6e2492d809861978bbf71</td>\n",
       "      <td>course title</td>\n",
       "      <td>Yes, even if you don't register, you're still ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">68a6f6eb226a7d51</th>\n",
       "      <th>0</th>\n",
       "      <td>e63e8853d1a83a9e6af773d79ea3f225</td>\n",
       "      <td>prerequisites</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e63e8853d1a83a9e6af773d79ea3f225</td>\n",
       "      <td>prerequisites</td>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e63e8853d1a83a9e6af773d79ea3f225</td>\n",
       "      <td>prerequisites</td>\n",
       "      <td>Solution:\\nCheck if you’re on the Developer Pl...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">def40c892a1aa7e7</th>\n",
       "      <th>0</th>\n",
       "      <td>c2afc1c738bb824d43e684500bc25aa6</td>\n",
       "      <td>grading system leaderboard</td>\n",
       "      <td>When you set up your account you are automatic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>c2afc1c738bb824d43e684500bc25aa6</td>\n",
       "      <td>grading system leaderboard</td>\n",
       "      <td>After you submit your homework it will be grad...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>c2afc1c738bb824d43e684500bc25aa6</td>\n",
       "      <td>grading system leaderboard</td>\n",
       "      <td>It is recommended by the Docker do\\n[Windows 1...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>c2afc1c738bb824d43e684500bc25aa6</td>\n",
       "      <td>grading system leaderboard</td>\n",
       "      <td>Cause:\\nIt happens because the apps are not up...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>c2afc1c738bb824d43e684500bc25aa6</td>\n",
       "      <td>grading system leaderboard</td>\n",
       "      <td>When you see this in logs, your container with...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">b1daa2166bc74902</th>\n",
       "      <th>0</th>\n",
       "      <td>82a2b9d50a621793aa28d17d576ddd95</td>\n",
       "      <td>creator and launch year of Data Engineering Zo...</td>\n",
       "      <td>Copy the file found in the Java example: data-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>82a2b9d50a621793aa28d17d576ddd95</td>\n",
       "      <td>creator and launch year of Data Engineering Zo...</td>\n",
       "      <td>Assuming you downloaded the Mage repo in the w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>82a2b9d50a621793aa28d17d576ddd95</td>\n",
       "      <td>creator and launch year of Data Engineering Zo...</td>\n",
       "      <td>There are 3 Zoom Camps in a year, as of 2024. ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>82a2b9d50a621793aa28d17d576ddd95</td>\n",
       "      <td>creator and launch year of Data Engineering Zo...</td>\n",
       "      <td>I initially followed data-engineering-zoomcamp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>82a2b9d50a621793aa28d17d576ddd95</td>\n",
       "      <td>creator and launch year of Data Engineering Zo...</td>\n",
       "      <td>This error appeared when running the command: ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">c3137462e600136d</th>\n",
       "      <th>0</th>\n",
       "      <td>1a9d367e2850d212195f35a561aafc09</td>\n",
       "      <td>Python version requirement</td>\n",
       "      <td>A generator is a function in python that retur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>1a9d367e2850d212195f35a561aafc09</td>\n",
       "      <td>Python version requirement</td>\n",
       "      <td>Yes, for simplicity (of troubleshooting agains...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>1a9d367e2850d212195f35a561aafc09</td>\n",
       "      <td>Python version requirement</td>\n",
       "      <td>confluent-kafka: `pip install confluent-kafka`...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>1a9d367e2850d212195f35a561aafc09</td>\n",
       "      <td>Python version requirement</td>\n",
       "      <td>Change the hadoop version to 3.0.1.Replace all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>1a9d367e2850d212195f35a561aafc09</td>\n",
       "      <td>Python version requirement</td>\n",
       "      <td>os.system(f\"curl -LO {url} -o {csv_name}\")</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">ef7fc00c37695c34</th>\n",
       "      <th>0</th>\n",
       "      <td>bd2ec99e47768403e3a9f4ec73dc689f</td>\n",
       "      <td>AWS S3 coverage in course</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>bd2ec99e47768403e3a9f4ec73dc689f</td>\n",
       "      <td>AWS S3 coverage in course</td>\n",
       "      <td>Yes, we will keep all the materials after the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>bd2ec99e47768403e3a9f4ec73dc689f</td>\n",
       "      <td>AWS S3 coverage in course</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>bd2ec99e47768403e3a9f4ec73dc689f</td>\n",
       "      <td>AWS S3 coverage in course</td>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>bd2ec99e47768403e3a9f4ec73dc689f</td>\n",
       "      <td>AWS S3 coverage in course</td>\n",
       "      <td>Yes, the slack channel remains open and you ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">e93733f35f608111</th>\n",
       "      <th>0</th>\n",
       "      <td>f59b7b20b0ccdb989ffa31cd67602ce2</td>\n",
       "      <td>Zoom link for weekly office hours</td>\n",
       "      <td>The zoom link is only published to instructors...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f59b7b20b0ccdb989ffa31cd67602ce2</td>\n",
       "      <td>Zoom link for weekly office hours</td>\n",
       "      <td>Yes! Every “Office Hours” will be recorded and...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f59b7b20b0ccdb989ffa31cd67602ce2</td>\n",
       "      <td>Zoom link for weekly office hours</td>\n",
       "      <td>We will probably have some calls during the Ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f59b7b20b0ccdb989ffa31cd67602ce2</td>\n",
       "      <td>Zoom link for weekly office hours</td>\n",
       "      <td>Pyspark converts the difference of two Timesta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f59b7b20b0ccdb989ffa31cd67602ce2</td>\n",
       "      <td>Zoom link for weekly office hours</td>\n",
       "      <td>It depends on your background and previous exp...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">6683e620f3b652fc</th>\n",
       "      <th>0</th>\n",
       "      <td>f6d73558151a20f8869a65a78f514cb9</td>\n",
       "      <td>refund policy course</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>f6d73558151a20f8869a65a78f514cb9</td>\n",
       "      <td>refund policy course</td>\n",
       "      <td>Yes, we will keep all the materials after the ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>f6d73558151a20f8869a65a78f514cb9</td>\n",
       "      <td>refund policy course</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>f6d73558151a20f8869a65a78f514cb9</td>\n",
       "      <td>refund policy course</td>\n",
       "      <td>You can start by installing and setting up all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>f6d73558151a20f8869a65a78f514cb9</td>\n",
       "      <td>refund policy course</td>\n",
       "      <td>Yes, even if you don't register, you're still ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">42a79c1d9735be7f</th>\n",
       "      <th>0</th>\n",
       "      <td>e3b893e48f987720566577ceb9471813</td>\n",
       "      <td>weekly schedule of live lectures</td>\n",
       "      <td>We will probably have some calls during the Ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>e3b893e48f987720566577ceb9471813</td>\n",
       "      <td>weekly schedule of live lectures</td>\n",
       "      <td>Either use conda or pip for managing venv, usi...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>e3b893e48f987720566577ceb9471813</td>\n",
       "      <td>weekly schedule of live lectures</td>\n",
       "      <td>provider \"google\" {\\nproject     = var.project...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>e3b893e48f987720566577ceb9471813</td>\n",
       "      <td>weekly schedule of live lectures</td>\n",
       "      <td>Change the hadoop version to 3.0.1.Replace all...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>e3b893e48f987720566577ceb9471813</td>\n",
       "      <td>weekly schedule of live lectures</td>\n",
       "      <td>Origin of Solution (Mage Slack-Channel): https...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0ac2e605aab39a74</th>\n",
       "      <th>0</th>\n",
       "      <td>5a4c890b19c9200b47fb8b7e9386d223</td>\n",
       "      <td>GCP vs other cloud providers</td>\n",
       "      <td>For uniformity at least, but you’re not restri...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>5a4c890b19c9200b47fb8b7e9386d223</td>\n",
       "      <td>GCP vs other cloud providers</td>\n",
       "      <td>You can do most of the course without a cloud....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>5a4c890b19c9200b47fb8b7e9386d223</td>\n",
       "      <td>GCP vs other cloud providers</td>\n",
       "      <td>You can easily forward the ports of pgAdmin, p...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>5a4c890b19c9200b47fb8b7e9386d223</td>\n",
       "      <td>GCP vs other cloud providers</td>\n",
       "      <td>for windows if you having trouble install SDK ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>5a4c890b19c9200b47fb8b7e9386d223</td>\n",
       "      <td>GCP vs other cloud providers</td>\n",
       "      <td>It’s very easy to manage your docker container...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                    context.trace_id  \\\n",
       "context.span_id  document_position                                     \n",
       "a11dcc2f37759fa2 0                  2f4acf21a7d6e2492d809861978bbf71   \n",
       "                 1                  2f4acf21a7d6e2492d809861978bbf71   \n",
       "                 2                  2f4acf21a7d6e2492d809861978bbf71   \n",
       "                 3                  2f4acf21a7d6e2492d809861978bbf71   \n",
       "                 4                  2f4acf21a7d6e2492d809861978bbf71   \n",
       "68a6f6eb226a7d51 0                  e63e8853d1a83a9e6af773d79ea3f225   \n",
       "                 1                  e63e8853d1a83a9e6af773d79ea3f225   \n",
       "                 2                  e63e8853d1a83a9e6af773d79ea3f225   \n",
       "def40c892a1aa7e7 0                  c2afc1c738bb824d43e684500bc25aa6   \n",
       "                 1                  c2afc1c738bb824d43e684500bc25aa6   \n",
       "                 2                  c2afc1c738bb824d43e684500bc25aa6   \n",
       "                 3                  c2afc1c738bb824d43e684500bc25aa6   \n",
       "                 4                  c2afc1c738bb824d43e684500bc25aa6   \n",
       "b1daa2166bc74902 0                  82a2b9d50a621793aa28d17d576ddd95   \n",
       "                 1                  82a2b9d50a621793aa28d17d576ddd95   \n",
       "                 2                  82a2b9d50a621793aa28d17d576ddd95   \n",
       "                 3                  82a2b9d50a621793aa28d17d576ddd95   \n",
       "                 4                  82a2b9d50a621793aa28d17d576ddd95   \n",
       "c3137462e600136d 0                  1a9d367e2850d212195f35a561aafc09   \n",
       "                 1                  1a9d367e2850d212195f35a561aafc09   \n",
       "                 2                  1a9d367e2850d212195f35a561aafc09   \n",
       "                 3                  1a9d367e2850d212195f35a561aafc09   \n",
       "                 4                  1a9d367e2850d212195f35a561aafc09   \n",
       "ef7fc00c37695c34 0                  bd2ec99e47768403e3a9f4ec73dc689f   \n",
       "                 1                  bd2ec99e47768403e3a9f4ec73dc689f   \n",
       "                 2                  bd2ec99e47768403e3a9f4ec73dc689f   \n",
       "                 3                  bd2ec99e47768403e3a9f4ec73dc689f   \n",
       "                 4                  bd2ec99e47768403e3a9f4ec73dc689f   \n",
       "e93733f35f608111 0                  f59b7b20b0ccdb989ffa31cd67602ce2   \n",
       "                 1                  f59b7b20b0ccdb989ffa31cd67602ce2   \n",
       "                 2                  f59b7b20b0ccdb989ffa31cd67602ce2   \n",
       "                 3                  f59b7b20b0ccdb989ffa31cd67602ce2   \n",
       "                 4                  f59b7b20b0ccdb989ffa31cd67602ce2   \n",
       "6683e620f3b652fc 0                  f6d73558151a20f8869a65a78f514cb9   \n",
       "                 1                  f6d73558151a20f8869a65a78f514cb9   \n",
       "                 2                  f6d73558151a20f8869a65a78f514cb9   \n",
       "                 3                  f6d73558151a20f8869a65a78f514cb9   \n",
       "                 4                  f6d73558151a20f8869a65a78f514cb9   \n",
       "42a79c1d9735be7f 0                  e3b893e48f987720566577ceb9471813   \n",
       "                 1                  e3b893e48f987720566577ceb9471813   \n",
       "                 2                  e3b893e48f987720566577ceb9471813   \n",
       "                 3                  e3b893e48f987720566577ceb9471813   \n",
       "                 4                  e3b893e48f987720566577ceb9471813   \n",
       "0ac2e605aab39a74 0                  5a4c890b19c9200b47fb8b7e9386d223   \n",
       "                 1                  5a4c890b19c9200b47fb8b7e9386d223   \n",
       "                 2                  5a4c890b19c9200b47fb8b7e9386d223   \n",
       "                 3                  5a4c890b19c9200b47fb8b7e9386d223   \n",
       "                 4                  5a4c890b19c9200b47fb8b7e9386d223   \n",
       "\n",
       "                                                                                input  \\\n",
       "context.span_id  document_position                                                      \n",
       "a11dcc2f37759fa2 0                                                       course title   \n",
       "                 1                                                       course title   \n",
       "                 2                                                       course title   \n",
       "                 3                                                       course title   \n",
       "                 4                                                       course title   \n",
       "68a6f6eb226a7d51 0                                                      prerequisites   \n",
       "                 1                                                      prerequisites   \n",
       "                 2                                                      prerequisites   \n",
       "def40c892a1aa7e7 0                                         grading system leaderboard   \n",
       "                 1                                         grading system leaderboard   \n",
       "                 2                                         grading system leaderboard   \n",
       "                 3                                         grading system leaderboard   \n",
       "                 4                                         grading system leaderboard   \n",
       "b1daa2166bc74902 0                  creator and launch year of Data Engineering Zo...   \n",
       "                 1                  creator and launch year of Data Engineering Zo...   \n",
       "                 2                  creator and launch year of Data Engineering Zo...   \n",
       "                 3                  creator and launch year of Data Engineering Zo...   \n",
       "                 4                  creator and launch year of Data Engineering Zo...   \n",
       "c3137462e600136d 0                                         Python version requirement   \n",
       "                 1                                         Python version requirement   \n",
       "                 2                                         Python version requirement   \n",
       "                 3                                         Python version requirement   \n",
       "                 4                                         Python version requirement   \n",
       "ef7fc00c37695c34 0                                          AWS S3 coverage in course   \n",
       "                 1                                          AWS S3 coverage in course   \n",
       "                 2                                          AWS S3 coverage in course   \n",
       "                 3                                          AWS S3 coverage in course   \n",
       "                 4                                          AWS S3 coverage in course   \n",
       "e93733f35f608111 0                                  Zoom link for weekly office hours   \n",
       "                 1                                  Zoom link for weekly office hours   \n",
       "                 2                                  Zoom link for weekly office hours   \n",
       "                 3                                  Zoom link for weekly office hours   \n",
       "                 4                                  Zoom link for weekly office hours   \n",
       "6683e620f3b652fc 0                                               refund policy course   \n",
       "                 1                                               refund policy course   \n",
       "                 2                                               refund policy course   \n",
       "                 3                                               refund policy course   \n",
       "                 4                                               refund policy course   \n",
       "42a79c1d9735be7f 0                                   weekly schedule of live lectures   \n",
       "                 1                                   weekly schedule of live lectures   \n",
       "                 2                                   weekly schedule of live lectures   \n",
       "                 3                                   weekly schedule of live lectures   \n",
       "                 4                                   weekly schedule of live lectures   \n",
       "0ac2e605aab39a74 0                                       GCP vs other cloud providers   \n",
       "                 1                                       GCP vs other cloud providers   \n",
       "                 2                                       GCP vs other cloud providers   \n",
       "                 3                                       GCP vs other cloud providers   \n",
       "                 4                                       GCP vs other cloud providers   \n",
       "\n",
       "                                                                            reference  \n",
       "context.span_id  document_position                                                     \n",
       "a11dcc2f37759fa2 0                  The purpose of this document is to capture fre...  \n",
       "                 1                  Yes, we will keep all the materials after the ...  \n",
       "                 2                  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "                 3                  You can start by installing and setting up all...  \n",
       "                 4                  Yes, even if you don't register, you're still ...  \n",
       "68a6f6eb226a7d51 0                  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "                 1                  You can start by installing and setting up all...  \n",
       "                 2                  Solution:\\nCheck if you’re on the Developer Pl...  \n",
       "def40c892a1aa7e7 0                  When you set up your account you are automatic...  \n",
       "                 1                  After you submit your homework it will be grad...  \n",
       "                 2                  It is recommended by the Docker do\\n[Windows 1...  \n",
       "                 3                  Cause:\\nIt happens because the apps are not up...  \n",
       "                 4                  When you see this in logs, your container with...  \n",
       "b1daa2166bc74902 0                  Copy the file found in the Java example: data-...  \n",
       "                 1                  Assuming you downloaded the Mage repo in the w...  \n",
       "                 2                  There are 3 Zoom Camps in a year, as of 2024. ...  \n",
       "                 3                  I initially followed data-engineering-zoomcamp...  \n",
       "                 4                  This error appeared when running the command: ...  \n",
       "c3137462e600136d 0                  A generator is a function in python that retur...  \n",
       "                 1                  Yes, for simplicity (of troubleshooting agains...  \n",
       "                 2                  confluent-kafka: `pip install confluent-kafka`...  \n",
       "                 3                  Change the hadoop version to 3.0.1.Replace all...  \n",
       "                 4                         os.system(f\"curl -LO {url} -o {csv_name}\")  \n",
       "ef7fc00c37695c34 0                  The purpose of this document is to capture fre...  \n",
       "                 1                  Yes, we will keep all the materials after the ...  \n",
       "                 2                  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "                 3                  You can start by installing and setting up all...  \n",
       "                 4                  Yes, the slack channel remains open and you ca...  \n",
       "e93733f35f608111 0                  The zoom link is only published to instructors...  \n",
       "                 1                  Yes! Every “Office Hours” will be recorded and...  \n",
       "                 2                  We will probably have some calls during the Ca...  \n",
       "                 3                  Pyspark converts the difference of two Timesta...  \n",
       "                 4                  It depends on your background and previous exp...  \n",
       "6683e620f3b652fc 0                  The purpose of this document is to capture fre...  \n",
       "                 1                  Yes, we will keep all the materials after the ...  \n",
       "                 2                  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "                 3                  You can start by installing and setting up all...  \n",
       "                 4                  Yes, even if you don't register, you're still ...  \n",
       "42a79c1d9735be7f 0                  We will probably have some calls during the Ca...  \n",
       "                 1                  Either use conda or pip for managing venv, usi...  \n",
       "                 2                  provider \"google\" {\\nproject     = var.project...  \n",
       "                 3                  Change the hadoop version to 3.0.1.Replace all...  \n",
       "                 4                  Origin of Solution (Mage Slack-Channel): https...  \n",
       "0ac2e605aab39a74 0                  For uniformity at least, but you’re not restri...  \n",
       "                 1                  You can do most of the course without a cloud....  \n",
       "                 2                  You can easily forward the ports of pgAdmin, p...  \n",
       "                 3                  for windows if you having trouble install SDK ...  \n",
       "                 4                  It’s very easy to manage your docker container...  "
      ]
     },
     "execution_count": 3,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from phoenix.session.evaluation import get_retrieved_documents\n",
    "retrieved_documents_df = get_retrieved_documents(px.Client(), project_name=\"our-rag-project\", timeout=None)\n",
    "retrieved_documents_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "id": "08065592",
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>input</th>\n",
       "      <th>output</th>\n",
       "      <th>reference</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>context.span_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>953950d641a5a79b</th>\n",
       "      <td>what is the course title of this course</td>\n",
       "      <td>The course title is **Data Engineering Zoomcam...</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eac8b87bf408d7d1</th>\n",
       "      <td>What are the prerequisites for the course?</td>\n",
       "      <td>The prerequisites for the **Data Engineering Z...</td>\n",
       "      <td>GitHub - DataTalksClub data-engineering-zoomca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74e5940a662d6a64</th>\n",
       "      <td>How does the grading system and leaderboard work?</td>\n",
       "      <td>The grading system for the **Data Engineering ...</td>\n",
       "      <td>When you set up your account you are automatic...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1b85458d1c8739f</th>\n",
       "      <td>Who created the Data Engineering Zoomcamp and ...</td>\n",
       "      <td>I couldn't find specific information on who cr...</td>\n",
       "      <td>Copy the file found in the Java example: data-...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7819d69b24d70940</th>\n",
       "      <td>What version of Python will be officially requ...</td>\n",
       "      <td>The officially recommended version of Python f...</td>\n",
       "      <td>A generator is a function in python that retur...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9e9913ae3f4dd513</th>\n",
       "      <td>Does the course cover AWS S3 in detail, or onl...</td>\n",
       "      <td>The information available did not specify deta...</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d5888d47c37c426d</th>\n",
       "      <td>Where can I find the Zoom link for weekly offi...</td>\n",
       "      <td>You can find the link to join the weekly offic...</td>\n",
       "      <td>The zoom link is only published to instructors...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5c47afa5a725ff1d</th>\n",
       "      <td>How do I request a refund if I can’t continue ...</td>\n",
       "      <td>The available information did not specify the ...</td>\n",
       "      <td>The purpose of this document is to capture fre...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4cbb4d9dfabd34dd</th>\n",
       "      <td>What is the expected weekly schedule of live l...</td>\n",
       "      <td>The details regarding the specific weekly sche...</td>\n",
       "      <td>We will probably have some calls during the Ca...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0486baa9a11c8c1d</th>\n",
       "      <td>Why does the course use GCP rather than other ...</td>\n",
       "      <td>The course primarily uses **Google Cloud Platf...</td>\n",
       "      <td>For uniformity at least, but you’re not restri...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                                              input  \\\n",
       "context.span_id                                                       \n",
       "953950d641a5a79b           what is the course title of this course    \n",
       "eac8b87bf408d7d1         What are the prerequisites for the course?   \n",
       "74e5940a662d6a64  How does the grading system and leaderboard work?   \n",
       "f1b85458d1c8739f  Who created the Data Engineering Zoomcamp and ...   \n",
       "7819d69b24d70940  What version of Python will be officially requ...   \n",
       "9e9913ae3f4dd513  Does the course cover AWS S3 in detail, or onl...   \n",
       "d5888d47c37c426d  Where can I find the Zoom link for weekly offi...   \n",
       "5c47afa5a725ff1d  How do I request a refund if I can’t continue ...   \n",
       "4cbb4d9dfabd34dd  What is the expected weekly schedule of live l...   \n",
       "0486baa9a11c8c1d  Why does the course use GCP rather than other ...   \n",
       "\n",
       "                                                             output  \\\n",
       "context.span_id                                                       \n",
       "953950d641a5a79b  The course title is **Data Engineering Zoomcam...   \n",
       "eac8b87bf408d7d1  The prerequisites for the **Data Engineering Z...   \n",
       "74e5940a662d6a64  The grading system for the **Data Engineering ...   \n",
       "f1b85458d1c8739f  I couldn't find specific information on who cr...   \n",
       "7819d69b24d70940  The officially recommended version of Python f...   \n",
       "9e9913ae3f4dd513  The information available did not specify deta...   \n",
       "d5888d47c37c426d  You can find the link to join the weekly offic...   \n",
       "5c47afa5a725ff1d  The available information did not specify the ...   \n",
       "4cbb4d9dfabd34dd  The details regarding the specific weekly sche...   \n",
       "0486baa9a11c8c1d  The course primarily uses **Google Cloud Platf...   \n",
       "\n",
       "                                                          reference  \n",
       "context.span_id                                                      \n",
       "953950d641a5a79b  The purpose of this document is to capture fre...  \n",
       "eac8b87bf408d7d1  GitHub - DataTalksClub data-engineering-zoomca...  \n",
       "74e5940a662d6a64  When you set up your account you are automatic...  \n",
       "f1b85458d1c8739f  Copy the file found in the Java example: data-...  \n",
       "7819d69b24d70940  A generator is a function in python that retur...  \n",
       "9e9913ae3f4dd513  The purpose of this document is to capture fre...  \n",
       "d5888d47c37c426d  The zoom link is only published to instructors...  \n",
       "5c47afa5a725ff1d  The purpose of this document is to capture fre...  \n",
       "4cbb4d9dfabd34dd  We will probably have some calls during the Ca...  \n",
       "0486baa9a11c8c1d  For uniformity at least, but you’re not restri...  "
      ]
     },
     "execution_count": 4,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "from phoenix.session.evaluation import get_qa_with_reference\n",
    "\n",
    "queries_df = get_qa_with_reference(px.Client(), project_name=\"our-rag-project\", timeout=None)\n",
    "queries_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "id": "165c8aef",
   "metadata": {},
   "outputs": [],
   "source": [
    "from phoenix.evals import (\n",
    "    HallucinationEvaluator,\n",
    "    OpenAIModel,\n",
    "    QAEvaluator,\n",
    "    RelevanceEvaluator,\n",
    "    run_evals,\n",
    ")\n",
    "\n",
    "eval_model = OpenAIModel(model=\"gpt-4\")\n",
    "relevance_evaluator = RelevanceEvaluator(eval_model)\n",
    "hallucination_evaluator = HallucinationEvaluator(eval_model)\n",
    "qa_evaluator = QAEvaluator(eval_model)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "id": "c8d620ea",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "run_evals |██████████| 48/48 (100.0%) | ⏳ 00:07<00:00 |  6.48it/s"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>score</th>\n",
       "      <th>explanation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>context.span_id</th>\n",
       "      <th>document_position</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">a11dcc2f37759fa2</th>\n",
       "      <th>0</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question asks for the 'course title'. Howe...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for the title of a cour...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a 'course title'. T...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for the 'course title'....</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for the title of a cour...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"3\" valign=\"top\">68a6f6eb226a7d51</th>\n",
       "      <th>0</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for 'prerequisites' but...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>relevant</td>\n",
       "      <td>1</td>\n",
       "      <td>The question is asking for 'prerequisites'. Th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>relevant</td>\n",
       "      <td>1</td>\n",
       "      <td>The question is 'prerequisites' which is vague...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">def40c892a1aa7e7</th>\n",
       "      <th>0</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about a grading system ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>relevant</td>\n",
       "      <td>1</td>\n",
       "      <td>The question asks about the grading system lea...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about a grading system ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about a grading system ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about a grading system ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">b1daa2166bc74902</th>\n",
       "      <th>0</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question asks for the creator and launch y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question asks for the creator and launch y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question asks for the creator and launch y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question asks for the creator and launch y...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for the creator and lau...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">c3137462e600136d</th>\n",
       "      <th>0</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the Python versio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>relevant</td>\n",
       "      <td>1</td>\n",
       "      <td>The question is asking about the Python versio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the Python versio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the Python versio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the Python versio...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">ef7fc00c37695c34</th>\n",
       "      <th>0</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the coverage of A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the coverage of A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The reference text is a GitHub link to a data ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the coverage of A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the coverage of A...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">e93733f35f608111</th>\n",
       "      <th>0</th>\n",
       "      <td>relevant</td>\n",
       "      <td>1</td>\n",
       "      <td>The question is asking for the Zoom link for w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for the Zoom link for w...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a Zoom link for wee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a Zoom link for wee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a Zoom link for wee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">6683e620f3b652fc</th>\n",
       "      <th>0</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for information about t...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the refund policy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The reference text is a GitHub link for a data...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the refund policy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking about the refund policy...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">42a79c1d9735be7f</th>\n",
       "      <th>0</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a weekly schedule o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a weekly schedule o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a weekly schedule o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a weekly schedule o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a weekly schedule o...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th rowspan=\"5\" valign=\"top\">0ac2e605aab39a74</th>\n",
       "      <th>0</th>\n",
       "      <td>relevant</td>\n",
       "      <td>1</td>\n",
       "      <td>The question is asking for a comparison betwee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a comparison betwee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a comparison betwee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a comparison betwee...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>unrelated</td>\n",
       "      <td>0</td>\n",
       "      <td>The question is asking for a comparison betwee...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                                        label  score  \\\n",
       "context.span_id  document_position                     \n",
       "a11dcc2f37759fa2 0                  unrelated      0   \n",
       "                 1                  unrelated      0   \n",
       "                 2                  unrelated      0   \n",
       "                 3                  unrelated      0   \n",
       "                 4                  unrelated      0   \n",
       "68a6f6eb226a7d51 0                  unrelated      0   \n",
       "                 1                   relevant      1   \n",
       "                 2                   relevant      1   \n",
       "def40c892a1aa7e7 0                  unrelated      0   \n",
       "                 1                   relevant      1   \n",
       "                 2                  unrelated      0   \n",
       "                 3                  unrelated      0   \n",
       "                 4                  unrelated      0   \n",
       "b1daa2166bc74902 0                  unrelated      0   \n",
       "                 1                  unrelated      0   \n",
       "                 2                  unrelated      0   \n",
       "                 3                  unrelated      0   \n",
       "                 4                  unrelated      0   \n",
       "c3137462e600136d 0                  unrelated      0   \n",
       "                 1                   relevant      1   \n",
       "                 2                  unrelated      0   \n",
       "                 3                  unrelated      0   \n",
       "                 4                  unrelated      0   \n",
       "ef7fc00c37695c34 0                  unrelated      0   \n",
       "                 1                  unrelated      0   \n",
       "                 2                  unrelated      0   \n",
       "                 3                  unrelated      0   \n",
       "                 4                  unrelated      0   \n",
       "e93733f35f608111 0                   relevant      1   \n",
       "                 1                  unrelated      0   \n",
       "                 2                  unrelated      0   \n",
       "                 3                  unrelated      0   \n",
       "                 4                  unrelated      0   \n",
       "6683e620f3b652fc 0                  unrelated      0   \n",
       "                 1                  unrelated      0   \n",
       "                 2                  unrelated      0   \n",
       "                 3                  unrelated      0   \n",
       "                 4                  unrelated      0   \n",
       "42a79c1d9735be7f 0                  unrelated      0   \n",
       "                 1                  unrelated      0   \n",
       "                 2                  unrelated      0   \n",
       "                 3                  unrelated      0   \n",
       "                 4                  unrelated      0   \n",
       "0ac2e605aab39a74 0                   relevant      1   \n",
       "                 1                  unrelated      0   \n",
       "                 2                  unrelated      0   \n",
       "                 3                  unrelated      0   \n",
       "                 4                  unrelated      0   \n",
       "\n",
       "                                                                          explanation  \n",
       "context.span_id  document_position                                                     \n",
       "a11dcc2f37759fa2 0                  The question asks for the 'course title'. Howe...  \n",
       "                 1                  The question is asking for the title of a cour...  \n",
       "                 2                  The question is asking for a 'course title'. T...  \n",
       "                 3                  The question is asking for the 'course title'....  \n",
       "                 4                  The question is asking for the title of a cour...  \n",
       "68a6f6eb226a7d51 0                  The question is asking for 'prerequisites' but...  \n",
       "                 1                  The question is asking for 'prerequisites'. Th...  \n",
       "                 2                  The question is 'prerequisites' which is vague...  \n",
       "def40c892a1aa7e7 0                  The question is asking about a grading system ...  \n",
       "                 1                  The question asks about the grading system lea...  \n",
       "                 2                  The question is asking about a grading system ...  \n",
       "                 3                  The question is asking about a grading system ...  \n",
       "                 4                  The question is asking about a grading system ...  \n",
       "b1daa2166bc74902 0                  The question asks for the creator and launch y...  \n",
       "                 1                  The question asks for the creator and launch y...  \n",
       "                 2                  The question asks for the creator and launch y...  \n",
       "                 3                  The question asks for the creator and launch y...  \n",
       "                 4                  The question is asking for the creator and lau...  \n",
       "c3137462e600136d 0                  The question is asking about the Python versio...  \n",
       "                 1                  The question is asking about the Python versio...  \n",
       "                 2                  The question is asking about the Python versio...  \n",
       "                 3                  The question is asking about the Python versio...  \n",
       "                 4                  The question is asking about the Python versio...  \n",
       "ef7fc00c37695c34 0                  The question is asking about the coverage of A...  \n",
       "                 1                  The question is asking about the coverage of A...  \n",
       "                 2                  The reference text is a GitHub link to a data ...  \n",
       "                 3                  The question is asking about the coverage of A...  \n",
       "                 4                  The question is asking about the coverage of A...  \n",
       "e93733f35f608111 0                  The question is asking for the Zoom link for w...  \n",
       "                 1                  The question is asking for the Zoom link for w...  \n",
       "                 2                  The question is asking for a Zoom link for wee...  \n",
       "                 3                  The question is asking for a Zoom link for wee...  \n",
       "                 4                  The question is asking for a Zoom link for wee...  \n",
       "6683e620f3b652fc 0                  The question is asking for information about t...  \n",
       "                 1                  The question is asking about the refund policy...  \n",
       "                 2                  The reference text is a GitHub link for a data...  \n",
       "                 3                  The question is asking about the refund policy...  \n",
       "                 4                  The question is asking about the refund policy...  \n",
       "42a79c1d9735be7f 0                  The question is asking for a weekly schedule o...  \n",
       "                 1                  The question is asking for a weekly schedule o...  \n",
       "                 2                  The question is asking for a weekly schedule o...  \n",
       "                 3                  The question is asking for a weekly schedule o...  \n",
       "                 4                  The question is asking for a weekly schedule o...  \n",
       "0ac2e605aab39a74 0                  The question is asking for a comparison betwee...  \n",
       "                 1                  The question is asking for a comparison betwee...  \n",
       "                 2                  The question is asking for a comparison betwee...  \n",
       "                 3                  The question is asking for a comparison betwee...  \n",
       "                 4                  The question is asking for a comparison betwee...  "
      ]
     },
     "execution_count": 6,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "retrieved_documents_relevance_df = run_evals(\n",
    "    evaluators=[relevance_evaluator],\n",
    "    dataframe=retrieved_documents_df,\n",
    "    provide_explanation=True,\n",
    "    concurrency=20,\n",
    ")[0]\n",
    "retrieved_documents_relevance_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "59e47f48",
   "metadata": {},
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "run_evals |██████████| 48/48 (100.0%) | ⏳ 00:38<00:00 |  1.25it/s\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>label</th>\n",
       "      <th>score</th>\n",
       "      <th>explanation</th>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>context.span_id</th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "      <th></th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>953950d641a5a79b</th>\n",
       "      <td>factual</td>\n",
       "      <td>0</td>\n",
       "      <td>The query asks for the title of the course. Th...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>eac8b87bf408d7d1</th>\n",
       "      <td>factual</td>\n",
       "      <td>0</td>\n",
       "      <td>The answer is factual because it accurately li...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>74e5940a662d6a64</th>\n",
       "      <td>factual</td>\n",
       "      <td>0</td>\n",
       "      <td>The answer provided is factual. It accurately ...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>f1b85458d1c8739f</th>\n",
       "      <td>factual</td>\n",
       "      <td>0</td>\n",
       "      <td>The answer is factual because it accurately re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>7819d69b24d70940</th>\n",
       "      <td>hallucinated</td>\n",
       "      <td>1</td>\n",
       "      <td>The answer states that the officially recommen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>9e9913ae3f4dd513</th>\n",
       "      <td>hallucinated</td>\n",
       "      <td>1</td>\n",
       "      <td>The answer is hallucinated because the referen...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>d5888d47c37c426d</th>\n",
       "      <td>factual</td>\n",
       "      <td>0</td>\n",
       "      <td>The answer is factual because it accurately re...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>5c47afa5a725ff1d</th>\n",
       "      <td>factual</td>\n",
       "      <td>0</td>\n",
       "      <td>The answer is factual because it correctly sta...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4cbb4d9dfabd34dd</th>\n",
       "      <td>hallucinated</td>\n",
       "      <td>1</td>\n",
       "      <td>The answer is hallucinated because it introduc...</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>0486baa9a11c8c1d</th>\n",
       "      <td>factual</td>\n",
       "      <td>0</td>\n",
       "      <td>The answer is factual because it accurately re...</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                         label  score  \\\n",
       "context.span_id                         \n",
       "953950d641a5a79b       factual      0   \n",
       "eac8b87bf408d7d1       factual      0   \n",
       "74e5940a662d6a64       factual      0   \n",
       "f1b85458d1c8739f       factual      0   \n",
       "7819d69b24d70940  hallucinated      1   \n",
       "9e9913ae3f4dd513  hallucinated      1   \n",
       "d5888d47c37c426d       factual      0   \n",
       "5c47afa5a725ff1d       factual      0   \n",
       "4cbb4d9dfabd34dd  hallucinated      1   \n",
       "0486baa9a11c8c1d       factual      0   \n",
       "\n",
       "                                                        explanation  \n",
       "context.span_id                                                      \n",
       "953950d641a5a79b  The query asks for the title of the course. Th...  \n",
       "eac8b87bf408d7d1  The answer is factual because it accurately li...  \n",
       "74e5940a662d6a64  The answer provided is factual. It accurately ...  \n",
       "f1b85458d1c8739f  The answer is factual because it accurately re...  \n",
       "7819d69b24d70940  The answer states that the officially recommen...  \n",
       "9e9913ae3f4dd513  The answer is hallucinated because the referen...  \n",
       "d5888d47c37c426d  The answer is factual because it accurately re...  \n",
       "5c47afa5a725ff1d  The answer is factual because it correctly sta...  \n",
       "4cbb4d9dfabd34dd  The answer is hallucinated because it introduc...  \n",
       "0486baa9a11c8c1d  The answer is factual because it accurately re...  "
      ]
     },
     "execution_count": 7,
     "metadata": {},
     "output_type": "execute_result"
    },
    {
     "name": "stderr",
     "output_type": "stream",
     "text": []
    }
   ],
   "source": [
    "hallucination_eval_df, qa_eval_df = run_evals(\n",
    "    dataframe=queries_df,\n",
    "    evaluators=[hallucination_evaluator, qa_evaluator],\n",
    "    provide_explanation=True,\n",
    "    concurrency=20,\n",
    ")\n",
    "hallucination_eval_df"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "id": "2159f9af",
   "metadata": {},
   "outputs": [],
   "source": [
    "from phoenix.trace import DocumentEvaluations, SpanEvaluations\n",
    "\n",
    "px.Client().log_evaluations(\n",
    "    SpanEvaluations(eval_name=\"Hallucination\", dataframe=hallucination_eval_df),\n",
    "    SpanEvaluations(eval_name=\"QA Correctness\", dataframe=qa_eval_df),\n",
    "    DocumentEvaluations(\n",
    "        eval_name=\"Retrieval Relevance\", dataframe=retrieved_documents_relevance_df\n",
    "    ),\n",
    ")"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "dtczoomcamp",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.13.2"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
